{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "import jieba\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('./data/train.csv')\n",
    "# test=pd.read_csv('../data/test_dataset.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 文本特征预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seg_words(text):\n",
    "    #去掉不在(所有中文、大小写字母、数字)中的非法字符\n",
    "    regex = re.compile(r'[^\\u4e00-\\u9fa5A-Za-z0-9]')\n",
    "    text = regex.sub(' ', text) # 将非法字符用‘ ’替代\n",
    "    text = text.strip() # 去掉前后的空格\n",
    "    word_list = jieba.cut(text, cut_all= False)\n",
    "    return word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\ASUS\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.607 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title_seg</th>\n",
       "      <th>content_seg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>范冰冰 出轨 被 偷拍   这才 是 真相</td>\n",
       "      <td>发布 误导 信息</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>马容 竟然 去 了 非诚 匆扰   一出场 24 盏灯 全灭   孟 爷爷 都 笑 了</td>\n",
       "      <td>标题 与 内容 不符     骗 点击     欺骗 阅读者</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>突发   武冈 西站 木材 市场 燃起 大火   浓烟滚滚 火势 巨大   现场</td>\n",
       "      <td>报道 内容 不 实   描述 事实 也 不食</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>辞职 交警   爆   出 贪   污 惊人 内幕   99   车主 都 不 知道   速看</td>\n",
       "      <td>谣言       不 实</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>美国 两块 让 国人 汗颜 的 告示牌   告诉 我们 离 文明 有多远</td>\n",
       "      <td>虚假 宣传     以偏概全</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Title_seg  \\\n",
       "0                            范冰冰 出轨 被 偷拍   这才 是 真相   \n",
       "1      马容 竟然 去 了 非诚 匆扰   一出场 24 盏灯 全灭   孟 爷爷 都 笑 了   \n",
       "2         突发   武冈 西站 木材 市场 燃起 大火   浓烟滚滚 火势 巨大   现场   \n",
       "3  辞职 交警   爆   出 贪   污 惊人 内幕   99   车主 都 不 知道   速看   \n",
       "4             美国 两块 让 国人 汗颜 的 告示牌   告诉 我们 离 文明 有多远   \n",
       "\n",
       "                      content_seg  \n",
       "0                        发布 误导 信息  \n",
       "1  标题 与 内容 不符     骗 点击     欺骗 阅读者  \n",
       "2          报道 内容 不 实   描述 事实 也 不食  \n",
       "3                    谣言       不 实  \n",
       "4                  虚假 宣传     以偏概全  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将 content 分词\n",
    "train['Title_seg'] = train['Title'].apply(lambda x: \" \".join(seg_words(str(x))))\n",
    "train['content_seg'] = train['Report Content'].apply(lambda x: \" \".join(seg_words(str(x))))\n",
    "# test['content_seg'] = test['content'].apply(lambda x:\" \".join(seg_words(str(x))))\n",
    "train[['Title_seg', 'content_seg']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这里是训练集和测试集都要做处理\n",
    "\n",
    "train[\"content_comment_2c_seg\"] = (\n",
    "    train[\"content\"].astype(str) + train[\"comment_2c\"].astype(str)\n",
    ").apply(lambda x: \" \".join(seg_words(str(x))))\n",
    "\n",
    "train[\"content_comment_all_seg\"] = (\n",
    "    train[\"content\"].astype(str) + train[\"comment_all\"].astype(str)\n",
    ").apply(lambda x: \" \".join(seg_words(str(x))))\n",
    "\n",
    "test[\"content_comment_2c_seg\"] = (\n",
    "    test[\"content\"].astype(str) + test[\"comment_2\"].astype(str)\n",
    ").apply(\n",
    "    lambda x: \" \".join(seg_words(str(x)))\n",
    ")  # 变量名和train不一样\n",
    "test[\"content_comment_all_seg\"] = (\n",
    "    test[\"content\"].astype(str) + test[\"comment_all\"].astype(str)\n",
    ").apply(lambda x: \" \".join(seg_words(str(x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(train_seg, test_seg, num_words = 10000):\n",
    "    '''\n",
    "        train_seg\n",
    "    '''\n",
    "    maxlen = train_seg.apply(lambda x:len(x)).max() # 最长有多少词\n",
    "    tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words= num_words)#tokenizer分词器类\n",
    "    tokenizer.fit_on_texts(train_seg) # 词条化，生成文本词典\n",
    "    sequences_train = tokenizer.texts_to_sequences(train_seg) #把句子转化为单词序列，这里已经编码了\n",
    "    train_sequence_pad = tf.keras.preprocessing.sequence.pad_sequences(sequences_train, maxlen= maxlen, value = 0.0)\n",
    "    #补齐，因为keras只能输入等长序列\n",
    "    #按照最大的长度进行补齐\n",
    "    word_index = tokenizer.word_index # word_index: 字典，将单词（字符串）映射为它们的排名或者索引。仅在调用fit_on_texts之后设置。\n",
    "   # word_index_inversed = {v:k for k,v in word_index.items()}\n",
    "    print('train_sequence_pad shape:',train_sequence_pad.shape)\n",
    "    # 处理test\n",
    "    sequences_test = tokenizer.texts_to_sequences(test_seg)\n",
    "    test_sequence_pad = tf.keras.preprocessing.sequence.pad_sequences(sequences_test, maxlen= maxlen, value = 0.0)\n",
    "    print('test_sequence_pad shape:',test_sequence_pad.shape)\n",
    "    return train_sequence_pad, test_sequence_pad, word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理content\n",
    "(\n",
    "    train_content_sequence_pad,\n",
    "    test_content_sequence_pad,\n",
    "    word_index_cotent,\n",
    ") = preprocess(train_seg=train[\"content_seg\"], test_seg=test[\"content_seg\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理content + comment_2c\n",
    "(\n",
    "    train_content_comment_2c_sequence_pad,\n",
    "    test_content_comment_2c_sequence_pad,\n",
    "    word_index_with_comment_2c,\n",
    ") = preprocess(\n",
    "    train_seg=train[\"content_comment_2c_seg\"], test_seg=test[\"content_comment_2c_seg\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理content + comment_all\n",
    "(\n",
    "    train_content_comment_all_sequence_pad,\n",
    "    test_content__comment_all_sequence_pad,\n",
    "    word_index_with_comment_all,\n",
    ") = preprocess(\n",
    "    train_seg=train[\"content_comment_all_seg\"], test_seg=test[\"content_comment_all_seg\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index_cotent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "np.save( '../data/train_content_sequence_pad.npy',train_content_sequence_pad)\n",
    "np.save(\"../data/test_content_sequence_pad.npy\", test_content_sequence_pad)\n",
    "np.save( \"../data/train_label.npy\",train[\"label\"].to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save( \"../data/test_text.npy\", test[\"content_seg\"].to_numpy())\n",
    "np.save( \"../data/train_text.npy\",\n",
    "    train[\n",
    "        [\"content_seg\", \"content_comment_2c_seg\", \"content_comment_all_seg\"]\n",
    "    ].to_numpy(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"label\"].to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 图片特征预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '../data/'\n",
    "train_images_dir = data_dir + '/train_images/' #这也是一种设置路径的方法，可以记住\n",
    "test_images_dir = data_dir + '/test_images/'\n",
    "train_file = data_dir + '/train.csv'\n",
    "test_file = data_dir + '/test_dataset.csv'\n",
    "\n",
    "train_images_filenames = [train_images_dir + filename for filename in os.listdir(train_images_dir)]\n",
    "test_images_filenames = [test_images_dir + filename for filename in os.listdir(test_images_dir)]\n",
    "\n",
    "#tfrecord_file = data_dir + '/train/train.tfrecords'\n",
    "# train_filenames = train_cat_filenames + train_dog_filenames\n",
    "\n",
    "FIG_SIZE = (10,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def preprocess_image(image, target_fig_size):\n",
    "    '''\n",
    "        input: image对象,通过tf.io.read_file读入\n",
    "        ouput:返回一个tftensor\n",
    "        help(tf.image.decode_image),自动识别图片类型，gif返回num_frames, height, width, 3], 其他返回[height, width, num_channels]\n",
    "        gif = tf.io.read_file('E:/dataset/2019-nCoV/competition/fake_news/train_images/02b6176b5e8dcd910112b2b63eb3f8bb.gif')\n",
    "        gif_tensor = tf.image.decode_image(gif, channels=3) 只取了第一帧\n",
    "        gif_tensor.shape # \n",
    "    '''\n",
    "    img_tensor = tf.image.decode_image(image, channels=3)\n",
    "    img_final = tf.image.resize(img_tensor, [*target_fig_size])\n",
    "    img_final /= 255.0  # normalize to [0,1] range\n",
    "    return img_final\n",
    "\n",
    "def load_and_preprocess_image(path, target_fig_size):\n",
    "    image = tf.io.read_file(path)\n",
    "    return preprocess_image(image,target_fig_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FIG_SIZE = (10,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#非常慢\n",
    "feature = tf.constant([], shape= [0,*FIG_SIZE,3],dtype='float32')\n",
    "count = []# 不可以写成 tf.float64?\n",
    "for i in range(train.shape[0]):\n",
    "    # 如果没有图片，就补充一个1，FIG_SIZE,3形状的全零张量\n",
    "    if i in train.loc[train.picture_lists.isna()].index:\n",
    "        zero_tensor= tf.zeros([1,*FIG_SIZE,3])\n",
    "        feature= tf.concat((feature,zero_tensor),axis = 0)\n",
    "        print(train.loc[i,'picture_lists'])\n",
    "        print(i,'done zero')\n",
    "        count.append(i)\n",
    "    else:\n",
    "        # 如果有图片，只取第一张图片进行处理\n",
    "        first_image_name = train.loc[i,'picture_lists'].split('\\t')[0]\n",
    "        first_image_path = train_images_dir + first_image_name\n",
    "        try:\n",
    "            print(i,first_image_path)\n",
    "            tf_tensor = load_and_preprocess_image(first_image_path, FIG_SIZE)\n",
    "            tf_tensor= tf.reshape(tf_tensor,[1,*FIG_SIZE, 3])\n",
    "            feature= tf.concat((feature,tf_tensor),axis = 0)\n",
    "            print(i,'done jpg')\n",
    "            count.append(i)\n",
    "        except:\n",
    "            feature= tf.concat((feature,zero_tensor),axis = 0)\n",
    "            print(i,first_image_path)\n",
    "            print(i,'done other')\n",
    "            count.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/train_image_feature.npy', feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_feature = tf.constant([], shape= [0,*FIG_SIZE,3],dtype='float32')\n",
    "count = []# 不可以写成 tf.float64?\n",
    "for i in range(test.shape[0]):\n",
    "    # 如果没有图片，就补充一个1，FIG_SIZE,3形状的全零张量\n",
    "    if i in test.loc[test.picture_lists.isna()].index:\n",
    "        zero_tensor= tf.zeros([1,*FIG_SIZE,3])\n",
    "        test_feature= tf.concat((test_feature,zero_tensor),axis = 0)\n",
    "        print(i,test.loc[i,'picture_lists'])\n",
    "        count.append(i)\n",
    "    else:\n",
    "        # 如果有图片，只取第一张图片进行处理\n",
    "        first_image_name = test.loc[i,'picture_lists'].split('\\t')[0]\n",
    "        first_image_path = test_images_dir + first_image_name\n",
    "        try:\n",
    "            tf_tensor = load_and_preprocess_image(first_image_path, FIG_SIZE)\n",
    "            tf_tensor= tf.reshape(tf_tensor,[1,*FIG_SIZE, 3])\n",
    "            test_feature= tf.concat((test_feature,tf_tensor),axis = 0)\n",
    "            print(i,first_image_path)\n",
    "            count.append(i)\n",
    "        except:\n",
    "            test_feature= tf.concat((test_feature,zero_tensor),axis = 0)\n",
    "            print(i,first_image_path)\n",
    "            count.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/test_image_feature.npy',test_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save( '../data/train_labels.npy', train[['ncw_label','fake_label','real_label']].to_numpy()) # train_label 是 one-hot形式的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 建模"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "import jieba\n",
    "import re\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('../data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['content'] = df.content.apply(lambda x: \" \".join(jieba.cut(x))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df[['content', 'ncw_label','fake_label','real_label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 100\n",
    "max_words = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=max_words)\n",
    "tokenizer.fit_on_texts(df1.content)\n",
    "sequences = tokenizer.texts_to_sequences(df1.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pad_sequences(sequences, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.array(df1[['ncw_label','fake_label','real_label']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = np.arange(data.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data = data[indices]\n",
    "labels = labels[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_samples = int(len(indices) * .8)\n",
    "validation_samples = len(indices) - training_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = data[:training_samples]\n",
    "y_train = labels[:training_samples]\n",
    "X_valid = data[training_samples: training_samples + validation_samples]\n",
    "y_valid = labels[training_samples: training_samples + validation_samples]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 加载训练好的词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zh_model = KeyedVectors.load_word2vec_format('../data/zh.vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(zh_model[next(iter(zh_model.vocab))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = len(zh_model[next(iter(zh_model.vocab))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = np.random.rand(max_words, embedding_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = (embedding_matrix - 0.5) * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for word, i in word_index.items():\n",
    "    if i < max_words:\n",
    "        try:\n",
    "          embedding_vector = zh_model.get_vector(word)\n",
    "          embedding_matrix[i] = embedding_vector\n",
    "        except:\n",
    "          pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 LSTM提取文本特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, Flatten, Dense, LSTM,Dropout\n",
    "\n",
    "units = 32\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(max_words, embedding_dim))\n",
    "model.add(LSTM(units))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(24,activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(3, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layers[0].set_weights([embedding_matrix])\n",
    "model.layers[0].trainable = False\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_acc', patience=10)\n",
    "history = model.fit(X_train, y_train,\n",
    "                    epochs=30,\n",
    "                    batch_size=32,\n",
    "                    validation_split=0.2,callbacks=[early_stop])\n",
    "model.save(\"mymodel.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "plt.plot(epochs, acc, label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_image_feature = np.load( '../data/train_image_feature.npy', allow_pickle=True)\n",
    "#test_image_feature = np.load('../data/test_image_feature.npy', allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#加载预训练模型\n",
    "train_content_sequence = keras.Input(\n",
    "    shape=(100,)\n",
    ")  # Variable-length sequence of ints\n",
    "Embedding_layer = keras.layers.Embedding(\n",
    "    max_words, embedding_dim, weights=[embedding_matrix], trainable=False\n",
    ")(train_content_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_text=tf.keras.layers.LSTM(16)(Embedding_layer)\n",
    "lstm_text= tf.keras.layers.Dense(8,activation='relu')(lstm_text)\n",
    "lstm_test_pr=tf.keras.layers.Dense(3,activation=\"softmax\")(lstm_text)\n",
    "text_model= keras.Model(inputs=train_content_sequence, outputs=lstm_test_pr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 CNN提取图片特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_input = keras.Input(shape=(10, 10, 3))  \n",
    "#image_feature = keras.layers.Dropout(0.2)(image_input)\n",
    "image_feature = keras.layers.Conv2D(15, kernel_size = 2, padding=\"same\",activation = 'relu')(image_input)\n",
    "image_feature = keras.layers.Dropout(0.2)(image_feature)\n",
    "image_feature = keras.layers.Conv2D(5, kernel_size = 2, activation = 'relu')(image_feature)\n",
    "image_feature = keras.layers.Dropout(0.2)(image_feature)\n",
    "image_feature = keras.layers.Conv2D(5, kernel_size = 2, activation = 'relu')(image_feature)\n",
    "image_feature = keras.layers.Dropout(0.2)(image_feature)\n",
    "#image_feature = keras.layers.MaxPooling2D(2)(image_feature)\n",
    "image_feature = keras.layers.Flatten()(image_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = keras.layers.Dense(8, activation=\"relu\")(image_feature)\n",
    "x = keras.layers.Dropout(0.2)(x)\n",
    "#x = keras.layers.Dense(8, activation=\"relu\")(x)\n",
    "im_pred = keras.layers.Dense(3, activation=\"softmax\", name=\"pred\")(x)\n",
    "img_model=keras.Model(inputs=image_input, outputs=im_pred)\n",
    "img_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 文本与图片融合多模态"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine wide and deep into one model\n",
    "merged_out = keras.layers.concatenate([text_model.output, img_model.output])\n",
    "merged_out=keras.layers.Dropout(0.2)(merged_out)\n",
    "merged_out =keras.layers.Dense(3, activation='softmax')(merged_out)\n",
    "\n",
    "combined_model = keras.Model([text_model.input,img_model.input], merged_out)\n",
    "print(combined_model.summary())\n",
    "\n",
    "combined_model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
    "              loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "early_stop = tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=10)\n",
    "history =  combined_model.fit([data, train_image_feature],\n",
    "                    labels, epochs=30, batch_size=32, validation_split=0.2,verbose=1,callbacks= [early_stop])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = history.history['accuracy']\n",
    "val_acc = history.history['val_accuracy']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "plt.plot(epochs, acc, label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.5 BiLSTM+ATTENTION多模态"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Bidirectional\n",
    "lstm_text1=keras.layers.Bidirectional(keras.layers.LSTM(16))(Embedding_layer)\n",
    "lstm_text1= keras.layers.Dense(8,activation='relu')(lstm_text1)\n",
    "attention=keras.layers.Attention(use_scale=False)([lstm_text1,lstm_text1])\n",
    "Input_layer=tf.keras.layers.Concatenate()([lstm_text1, attention])\n",
    "lstm_test_pr=keras.layers.Dense(3,activation=\"softmax\")(lstm_text1)\n",
    "text_model1= keras.Model(inputs=train_content_sequence ,outputs=lstm_test_pr)\n",
    "text_model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_input = keras.Input(shape=(10, 10, 3))  \n",
    "#image_feature = keras.layers.Dropout(0.2)(image_input)\n",
    "image_feature = keras.layers.Conv2D(15, kernel_size = 2, padding=\"same\",activation = 'relu')(image_input)\n",
    "image_feature = keras.layers.Dropout(0.2)(image_feature)\n",
    "image_feature = keras.layers.Conv2D(5, kernel_size = 2, activation = 'relu')(image_feature)\n",
    "image_feature = keras.layers.Dropout(0.2)(image_feature)\n",
    "image_feature = keras.layers.Conv2D(5, kernel_size = 2, activation = 'relu')(image_feature)\n",
    "image_feature = keras.layers.Dropout(0.2)(image_feature)\n",
    "image_feature = keras.layers.MaxPooling2D(2)(image_feature)\n",
    "image_feature = keras.layers.Flatten()(image_feature)\n",
    "x = keras.layers.Dense(8, activation=\"relu\")(image_feature)\n",
    "x = keras.layers.Dropout(0.2)(x)\n",
    "#x = keras.layers.Dense(8, activation=\"relu\")(x)\n",
    "im_pred = keras.layers.Dense(3, activation=\"softmax\", name=\"pred\")(x)\n",
    "img_model=keras.Model(inputs=image_input, outputs=im_pred)\n",
    "img_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine wide and deep into one model\n",
    "merged_out = keras.layers.concatenate([text_model1.output, img_model.output])\n",
    "merged_out =keras.layers.Dense(3, activation='softmax')(merged_out)\n",
    "combined_model = keras.Model([text_model1.input,img_model.input], merged_out)\n",
    "print(combined_model.summary())\n",
    "\n",
    "combined_model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
    "              loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "early_stop = tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=10)\n",
    "history =  combined_model.fit([data, train_image_feature],\n",
    "                    labels, epochs=30, batch_size=32, validation_split=0.2,verbose=1,callbacks= [early_stop])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = history.history['accuracy']\n",
    "val_acc = history.history['val_accuracy']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "plt.plot(epochs, acc, label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
